package com.g2room.driving.grap.article.qg;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrabNrjDataByProxy {
	String imgPath = "nrj";
	public String baseUrl = "http://www.mimito.com.cn";
	private String soruce = "女人街";
	private int count = 0;
	static List<String> titles = GenericArti.getTitles();
	
	public static void main(String[] args) throws Exception{
		GrabNrjDataByProxy gkd = new GrabNrjDataByProxy();
//		gkd.getNav();

        List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), gkd.soruce);
		// 抓取图片1446
		if(null != ids && ids.size() > 0) {
			HandleImages.getThumbImgs(ids, gkd.baseUrl, gkd.imgPath);
		}
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
		pages.put("http://www.mimito.com.cn/qinggan/page_015.html", "情感");
		pages.put("http://www.mimito.com.cn/sex-love/page_100.html", "性爱");

        for(String key : pages.keySet()) {
	        String pageTotal = key.substring(key.indexOf("page_") + 5, key.lastIndexOf("."));
	        for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
	        	String url = i==1 ? key.substring(0, key.indexOf("/page")) + "/index.html" : key.replaceAll("page_" + pageTotal, "page_" + i);
	        	boolean rs = true;
	        	String tn = pages.get(key);
	        	if(tn.equals("性爱")) {
	        		rs = getSexPage(url, tn);
	        	} else if(tn.equals("情感")) {
		        	rs = getQgPage(url, tn);
	        	}
	        	if(!rs) break;
	        }
        }
        System.out.println("本次抓取数据：" + count);
		
        List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), soruce);
		// 抓取图片
		if(null != ids && ids.size() > 0) {
			HandleImages.getContentImgs(ids, baseUrl, imgPath);
//			HandleImages.getThumbImgs(ids, baseUrl, imgPath);
		}
		// 最后输出失败的链接到数据库
		GenericArti.insertError2Db(soruce);
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getSexPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		Document doc = GenericGrap.getDoc(url);
		Elements els = doc.getElementsByAttributeValue("class", "travel");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				Element hrefEl = el.getElementsByTag("h2").get(0).getElementsByTag("a").get(0);
				String title = hrefEl.text();
				if(GenericArti.titles.contains(title)) {
					System.out.println("存在，跳过：" + title);
					if(artis.size() == 0) {
						// 后续抓取的时候，就存在则直接返回
						return false;
					}
				}
				Elements thumbEls = el.getElementsByTag("img");
				Element thumbEl = null;
				ArticlesTo arti = new ArticlesTo();
				if(thumbEls != null && thumbEls.size() > 0) {
					thumbEl = el.getElementsByTag("img").get(0);
					arti.setThumb(baseUrl + thumbEl.attr("src"));
				}
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(hrefEl.attr("href"));
				arti.setTypeName(typeName);
				arti.setState(8);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}
	

	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getQgPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		Document doc = GenericGrap.getDoc(url);
		Elements els = doc.getElementById("n1").getElementsByTag("li");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				Element hrefEl = el.getElementsByTag("a").get(0);
				String title = hrefEl.text();
				if(GenericArti.titles.contains(title)) {
					System.out.println("存在，跳过：" + title);
					if(artis.size() == 0) {
						// 后续抓取的时候，就存在则直接返回
						return false;
					} else {
						break;
					}
				}
				Element timeEl = el.getElementsByAttributeValue("class", "time").get(0);
				ArticlesTo arti = new ArticlesTo();
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(hrefEl.attr("href"));
				arti.setTypeName(typeName);
				arti.setPublishTime(timeEl.text());
				arti.setState(8);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}
	
	public void getDetail(ArticlesTo arti) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + arti.getSourceUrl());
		try {
			Document doc = GenericGrap.getDoc(arti.getSourceUrl());
			// 获取分页信息
			Elements els = doc.getElementsByAttributeValue("class", "page2").get(0).getElementsByTag("a");
			// 删除广告div
			doc.getElementsByAttributeValue("class", "t5").remove();
			String content = doc.getElementsByAttributeValue("class", "content_01").html();
			if(null != els && els.size()>2) {
				// 去掉当前页和下一页
				els.remove(els.size() - 1);
				els.remove(0);
				for(Element el : els) {
					String href = el.attr("href");
					Document pageDoc = GenericGrap.getDoc(href);
					// 删除广告div
					pageDoc.getElementsByAttributeValue("class", "t5").remove();
					content += "\n"+pageDoc.getElementsByAttributeValue("class", "content_01").html();
				}
			}
			content = GenericArti.handleSpace(HtmlUtil.delAllTag(content));
			
			arti.setContent("<div class=\"article\">" + content + "</div>");
			// 获取发布时间
			String info = doc.getElementById("artinfo").html();
			String publishTime = info.substring(info.indexOf("20"), info.indexOf("20") + 10);
			arti.setPublishTime(publishTime);
			GenericArti.insert2Db(arti);
			GenericArti.titles.add(arti.getTitle());
			count++;
        } catch (Exception e) {
        	GenericArti.errorUrls.put(arti.getSourceUrl(), arti.getTypeName());
        	e.printStackTrace();
        	return;
        }
	}
}
